'''
Created on Mar 9, 2013

@author: arenduchintala
'''

import sys, os, re
import locale
wordCounts = {}
emissionCounts = {}

def replaceRareWord(token):
    try:
        locale.atof(token)
        return '_RARE_NUMERIC_'
    except:
        if  len(re.findall(r'[A-Z]', token)) == len(token):
            return '_RARE_ALL_CAPS_'
        elif len(re.findall(r'[A-Z]', token[0])) == 1:
            return '_RARE_FIRST_CAP_'
        elif (token.endswith('s')):
            return  '_RARE_END_S_'
        elif len(re.findall(r'\-', token)) > 0:
            return '_RARE_HAS_DASH_'
        else:
            return  '_RARE_'
        
def parseCountFile(inputFile):
    lines = open(inputFile, 'r').readlines()
    for line in lines:
        items = line.strip().split(" ")
        if (items[1].strip() == 'WORDTAG'):
            emissionCounts[items[2] + ',' + items[3]] = int(items[0])
            if wordCounts.has_key(items[3]):
                wordCounts[items[3]] = wordCounts[items[3]] + int(items[0])
            else:
                wordCounts[items[3]] = int(items[0])
                
    
if __name__ == "__main__":
    locale.setlocale( locale.LC_ALL, 'en_US.UTF-8' )
    parseCountFile('../conll/pos.count')
    originalFile = open('../conll/pos.train', 'r')
    originalTrainingLines = originalFile.readlines()
    modLines = originalTrainingLines
    originalFile.close()
    
    writer = open('../conll/pos-rare.train', 'w')
    for i in range (len(originalTrainingLines)):
        
        if  originalTrainingLines[i] != '\n':
           
            originalTrainingLines[i] = originalTrainingLines[i].strip()
            currentToken = originalTrainingLines[i].split(" ")[0].strip()
            currentTokenTag = originalTrainingLines[i].split(" ")[1].strip()
            if (wordCounts[currentToken] < 2):
                modLines[i] = replaceRareWord(currentToken) + " " + currentTokenTag + '\n'
            else:
                modLines[i] = currentToken + ' ' + currentTokenTag + '\n'
        else:
            modLines[i] = '\n'
        writer.write(modLines[i])
    writer.flush();
    writer.close();
    os.system("python count_freqs.py ../conll/pos-rare.train > ../conll/pos-rare.count")
    #os.system("python trigram-hmm-tagger.py")